package com.aroura.mycrawler;

import us.codecraft.webmagic.*;
import us.codecraft.webmagic.model.annotation.HelpUrl;
import us.codecraft.webmagic.model.annotation.TargetUrl;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;

import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.regex.Matcher;
import java.util.regex.Pattern;


public class BlogPageProcessor implements PageProcessor {
    //抓取网站的相关配置，包括：编码、抓取间隔、重试次数等
    private Site site = Site.me().setRetryTimes(10)
            .setSleepTime(500)
            .setTimeOut(30000)
            .setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36");

    //抓取博文数量
    private static int num = 0;

    @Override
    public void process(Page page) {
        //1.如果是博文列表页(博文入口页面)，就将详细博文页面url添加到目标target集合中，并添加下一页的url放到target集合里
        if (page.getUrl().regex("https://blog\\.hellobi\\.com/hot/weekly\\?page=\\d+").match()) {
            //目标链接
            page.addTargetRequests(page.getHtml().xpath("//h2[@class='title']/a").links().all());
            //下一页
            page.addTargetRequests(page.getHtml().xpath("//a[@rel='next']").links().all());
        } else {
            //2.如果是博文详情页，提取页面信息
            String title = page.getHtml().xpath("//h1[@class='clearfix']/a/text()").get();
            String url = page.getHtml().xpath("//h1[@class='clearfix']/a/@href").get();
            String author = page.getHtml().xpath("//section[@class='sidebar']/div/div/a[@class='aw-user-name']/text()").get();
            String blogHomeUrl = page.getHtml().xpath("//section[@class='sidebar']/div/div/a[@class='aw-user-name']/@href").get();
            String content = page.getHtml().xpath("//div[@class='message-content editor-style']").get();
            String recommendNum = page.getHtml().xpath("//a[@class='agree']/b/text()").get();
            String commentNum = page.getHtml().xpath("//div[@class='aw-mod']/div/h2/text()").get();
            String readNum = page.getHtml().xpath("//div[@class='row']/div/div/div/div/span/text()").get();
            System.out.println("rawTime: " + page.getHtml().xpath("//time[@class='time']/text()").get());
            String rawTime = page.getHtml().xpath("//time[@class='time']/text()").get().split(":")[1].trim();
            String publishTime = convertToDateString(rawTime);
            BlogInfo blogInfo = new BlogInfo();
            blogInfo.setTitle(title);
            blogInfo.setUrl(url);
            blogInfo.setAuthor(author);
            blogInfo.setBlogHomeUrl(blogHomeUrl);
            blogInfo.setContent(content);
            blogInfo.setRecommendNum(recommendNum);
            blogInfo.setCommentNum(commentNum);
            blogInfo.setReadNum(readNum);
            blogInfo.setPublishTime(publishTime);
            page.putField("blogInfo", blogInfo);
        }

    }

    @Override
    public Site getSite() {
        return this.site;
    }

    public static void main(String[] args) {
        Spider.create(new BlogPageProcessor())
                .addPipeline(new Pipeline() {
                    @Override
                    public void process(ResultItems resultItems, Task task) {
                        BlogInfo blogInfo = resultItems.get("blogInfo");
                        System.out.println(blogInfo);
                    }
                })
                .addUrl("https://blog.hellobi.com/hot/weekly?page=1")
                .thread(5)
                .run();
    }

    private String convertToDateString(String rawDate) {
        String publishTime = null;
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
        Calendar cal = Calendar.getInstance();
        Pattern p = Pattern.compile("^\\d{4}-\\d{2}-\\d{2}$");
        Matcher matcher = p.matcher(rawDate);
        if (matcher.matches()) {
            publishTime = rawDate;
        } else if (rawDate.contains("天")) {
            int days = Integer.parseInt(rawDate.split("天")[0].trim());
            cal.add(Calendar.DAY_OF_MONTH, -days);
            publishTime = sdf.format(cal.getTime());
        } else {
            publishTime = sdf.format(cal.getTime());
        }
        return publishTime;
    }
}
